#include "../../include/PageLib.hpp"

using namespace tinyxml2;
namespace se
{

std::vector<std::string> & DirScanner::getFiles()
{
    return _files;
}

void DirScanner::traverse(std::string dir)
{
    DIR *pdir;
    struct dirent *ptr;
    if((pdir = opendir(dir.c_str())) == NULL)
    {
        perror("opendir");
        return;
    }
    while((ptr = readdir(pdir)) != NULL)
    {
        if(strcmp(ptr->d_name, ".") == 0 || strcmp(ptr->d_name, "..") == 0)
        {
            continue;
        }
        else if(ptr->d_type == DT_REG)
        {
            std::string strFile;
            strFile = dir;
            strFile += "/";
            strFile += ptr->d_name;
            _files.push_back(strFile);
            continue;
        }
        else
        {
            continue;
        }
    }
    closedir(pdir);
    return;
}

std::string FileProcessor::process(std::string fileName)
{
    return _titleFeature;//待重写，我不会
}

PageLib::PageLib(DirScanner & dirScanner)//构造函数 自己写的
    :_dirScanner(dirScanner)
    ,_files()
     ,_offsetLib()
{
    _dirScanner.traverse("../../config/人民网语料");
}
/* PageLib::PageLib(Configuration & conf, DirScanner & dirScanner, FileProcessor & fileProcessor)//老师给的 构造函数 */
/*     :_dirScanner(dirScanner) */
/*     ,_files() */
/*      ,_offsetLib() */
/* { */
/*     _dirScanner.traverse(conf._filepath); */
/* } */

<<<<<<< HEAD
struct RssItem
{
    string title;
    string link;
    string description;
};

void filter(string &s)
{
    regex table_re1("<span.*?>.*?</span>");
    regex table_re2("<[^<]*>");
    s = regex_replace(s,table_re1,"");
    s = regex_replace(s,table_re2,"");
}

=======
>>>>>>> aee1d99473ab65825124d6358bc171a552109ecf
void PageLib::create()//创建网页库
{

    XMLDocument webDoc;
    std::vector<std::string> filenames = _dirScanner.getFiles();
    for(auto & filename : filenames)
    {
        webDoc.LoadFile(filename.c_str());
        XMLElement* pRss = webDoc.RootElement();
        XMLElement* pChannel = pRss->FirstChildElement( "channel" );
        XMLElement* pItem = pChannel->FirstChildElement( "item" );

        while(pItem)
        {
            struct RssItem rssItem;
            string newfile;

            XMLElement* pTitle = pItem->FirstChildElement( "title" );
            rssItem.title = pTitle->GetText();
            filter(rssItem.title);
            /* cout << rssItem.title << endl; */

            XMLElement* pLink = pTitle->NextSiblingElement( "link" );
            rssItem.link = pLink->GetText();
            filter(rssItem.link);
            /* cout << rssItem.link << endl; */

            XMLElement* pDescription = pLink->NextSiblingElement( "description" );
            if(pDescription)
            {
                rssItem.description = pDescription->GetText();
                filter(rssItem.description);
            }
            /* cout << rssItem.description << endl; */

            newfile = " <title> " + rssItem.title + " </title> <link> " 
                + rssItem.link + " </link> <description> " 
                + rssItem.description + " </description> ";
            _files.push_back(newfile);

            pItem = pItem->NextSiblingElement();
        }
        /* cout << "webDoc.ErrorID" << webDoc.ErrorID() << endl; */
    }
}

void PageLib::store()//存储网页库和位置偏移库
{
    ofstream ofs1("../../config/WebLibFile/ripepage.dat");
    if(!ofs1.good())
    {
        cout << "ofstream not good" << endl;
        return;
    }
    ofstream ofs2("../../config/WebLibFile/offset.dat");
    if(!ofs2.good())
    {
        cout << "ofstream not good" << endl;
        return;
    }

    int v_cnt = 0;
    for(auto & file :_files)
    {
        ++v_cnt;
        int position1 = ofs1.tellp();

        ofs1 << "<doc> <docid>" << v_cnt <<
            "</docid>"<< file << "</doc>" << endl;
        int position2 = ofs1.tellp();
        int length = position2 - position1;
        _offsetLib.insert(make_pair(v_cnt, make_pair(position1, length)));
    }

    for(auto & offset :_offsetLib)
    {
        ofs2 << offset.first << " " << offset.second.first << " " << offset.second.second << endl;
    }

    ofs1.close();
    ofs2.close();
}


}//end of namespace se

#if 0

using namespace se;
int main()
{
    DirScanner dirScanner;
    PageLib pageLib(dirScanner);
    pageLib.create();
    pageLib.store();
}

#endif

